import java.sql.*;

import javax.sql.*;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.BufferedWriter;
import java.io.InputStreamReader;

import java.io.IOException;

import java.util.regex.*;

import java.lang.*;
import java.util.*;
import java.util.Date;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.lang.StringEscapeUtils;

public class GetChangeStatistics 
{
	//the file to store distribution information
	String changeDstFile = null;
	String targetDstFile = null;
	String messageDstFile = null;
	
	//db object
	DB db = null;
	
	//threshold timestamp, used to indicate whether the time is null or not
	Timestamp thresholdTime = null;
	
	//the number of rows fetched each time
	public final static int CHANGEBORY_ROW_LIMIT= 1000;
	
	//the lines of message fetched each time
	public final static int MESSAGE_ROW_LIMIT= 1000;
	
	//the number of rows fetched each time
	public final static int BUGID_ROW_LIMIT= 10000;

	//store change entity num distributed map
	private HashMap<Integer, Integer> changePDFMap = null;
	
	//store target entity num distributed map
	private HashMap<Integer, Integer> targetPDFMap = null;
	
	//store message entity num distributed map
	private HashMap<Integer, Integer> messagePDFMap = null;
	
	
	//prepared sql statement
	PreparedStatement getLinesPs = null;
	PreparedStatement getTargetPs = null;
	PreparedStatement getMessagePs = null;
	
	//store bugid
	private HashSet<Integer> bugidSet = null;
	
	//store message id
	private HashSet<Integer> bugidReferencedSet = null;
	private HashSet<String> commitHashSet = null;
	
	//to analyze bufid reference
	Matcher bugidMatch = null;
	
	public GetChangeStatistics() throws ClassNotFoundException, 
		IllegalAccessException, SQLException
	{	
		//connect to the db
		db = new DB();
		
		db.connect();
		
		String getTargetStatement = "select line from target where commitHash = ?";
		getTargetPs = db.getDynamicCheckPreparedStatement(getTargetStatement);
		
		//bugid regex object
		String bugidRegex = "(\\W|^)+bug(\\s*|\\W)+(\\d{1,5})($|\\W)+?";
		
		Pattern bugidPattern = Pattern.compile(bugidRegex, Pattern.CASE_INSENSITIVE);
		
		bugidMatch = bugidPattern.matcher("");
		
		
		/*
		//set up threshold timestamp
		
		//set up a timezone
		SimpleTimeZone tZone = new SimpleTimeZone(0, "");		
		
		Calendar dayTime = Calendar.getInstance();
		
		dayTime.setTimeZone(tZone);
		
		//set up the threshold time as 2012-Jan-20
		dayTime.set(2012, 1, 20, 0, 0, 0);
		
		//convert it into millisec since the GMT time and set up the timestamp
		thresholdTime = new Timestamp(dayTime.getTime().getTime());
		*/
		
	}
	
	/*
	//check whether this timestamp is newer than the threshold time
	private boolean isTimeNull(Timestamp time)
	{
		return (time.getTime() - thresholdTime.getTime() > 0);
	}
	*/
	
	/*
	private void getChangeEntityNum(String changeDstFile, String messageDstFile) 
			throws SQLException, IOException
	{

		//to store distribution information into this file
		this.changeDstFile = changeDstFile;
		//this.targetDstFile = targetDstFile;
		this.messageDstFile = messageDstFile;
		
		//instantiate the distribution hash map
		changePDFMap = new HashMap<Integer, Integer>();
		//targetPDFMap = new HashMap<Integer, Integer>();
		messagePDFMap = new HashMap<Integer, Integer>();
		
		//load bugid list from table bugBody
		loadBugid(BUGID_ROW_LIMIT);
		
		//instantiate the bugid set
		bugidReferencedSet = new HashSet<Integer>();
		commitHashSet = new HashSet<String>();
		
		
		int count = 0;
		int totalEntityNum = 0;
				
		//retrive bugBody
		ResultSet ChangeBodyRs = getChangeBodyPs.executeQuery();
		while (ChangeBodyRs.next())
		{
			//there are 11 unchanged entities
			int entityNum = 0;		
			
			
			String commitHash = ChangeBodyRs.getString(2);
			//cant be nykk
			if (commitHash == null)
			{
				throw new RuntimeException();
			}
			entityNum++;
			
			
			String project =  ChangeBodyRs.getString(1);
			if (project != null)
			{
				entityNum++;
			}
			
			String treeHash =  ChangeBodyRs.getString(3);
			if (treeHash != null)
			{
				entityNum++;
			}
			
			String parentHashes =  ChangeBodyRs.getString(4);
			if (parentHashes != null)
			{
				entityNum++;
			}
			
			String authorName =  ChangeBodyRs.getString(5);
			if (authorName != null)
			{
				entityNum++;
			}
			
			String authorEmail =  ChangeBodyRs.getString(6);
			if (authorEmail != null)
			{
				entityNum++;
			}
			
			Timestamp authorDate =  ChangeBodyRs.getTimestamp(7);
			if (!isTimeNull(authorDate))
			{
				entityNum++;
			}
			
			String committerName =  ChangeBodyRs.getString(8);
			if (committerName != null)
			{
				entityNum++;
			}
			
			String committerEmail =  ChangeBodyRs.getString(9);
			if (committerEmail != null)
			{
				entityNum++;
			}
			
			Timestamp committerDate =  ChangeBodyRs.getTimestamp(10);
			if (!isTimeNull(committerDate))
			{
				entityNum++;
			}
			
			String subject =  ChangeBodyRs.getString(11);
			if (subject != null)
			{
				entityNum++;
			}
			
			

			//get the target sub entities information
			//entityNum += getTargetStatistics(commitHash);
			entityNum++;

			//get the message sub entities information
			//entityNum += getTargetStatistics(commitHash);
			entityNum++;
			
			addMap(entityNum, changePDFMap);
		}
		
		System.out.println("change entities num --> " + totalEntityNum);
		
		writeDistributionToFile(changeDstFile, changePDFMap);
		writeDistributionToFile(messageDstFile, changePDFMap);
	}
	*/
	
	private void getLinesStatistics(int rowNumLimt, String messageDstFile) 
			throws SQLException, IOException
	{
		//load bugid list from table bugBody
		loadBugid(BUGID_ROW_LIMIT);
		
		
		//set up the prepared statement given corresponding sql statements
		String getLinesStatement = "select * from message " +
				"where line is not null";
		getLinesPs = db.getStreamPreparedStatement(getLinesStatement, rowNumLimt); 
		
		
		int referencedBugidCount = 0;
		
		//HashSet<String> commitHashSet = new HashSet<String>();
		bugidReferencedSet = new HashSet<Integer>();
		
		//HashMap<String, Integer>lineMap = new HashMap<String, Integer>();
		
		//HashMap<Integer, Integer>lineDistributionMap = 
				//new HashMap<Integer, Integer>();
		
		//String previousCommitHash = null;
		//int commitHashNum = 0;
		
		//retrieve bugBody
		ResultSet linesRs = getLinesPs.executeQuery();
		while (linesRs.next())
		{
			
			String commitHash = linesRs.getString(1);
			
			String line = linesRs.getString(2);	
			/*
			if (line.equals(previousCommitHash))
			{
				commitHashNum++;
			}
			else
			{
				if (previousCommitHash != null)
				{
					addMap(commitHashNum, lineDistributionMap);
				}
				
				commitHashNum = 1;
			}
			
			previousCommitHash = line;
			*/
			
			//addMap(commitHash, lineMap);
			
			bugidMatch.reset(line);
			while (bugidMatch.find())
			{				
				String idString = bugidMatch.group(bugidMatch.groupCount() - 1);
				
				if (idString != null)
				{
					//lookup the bugid condictionary
					int bugid = Integer.parseInt(idString);
				
					if (bugidSet.contains(bugid))
					{
						referencedBugidCount++;
						
						bugidReferencedSet.add(bugid);
						
						commitHashSet.add(commitHash);
						
						if (referencedBugidCount % 40 == 0)
						{
							System.out.println(bugid + " " + bugidMatch.group(0));
						}
					}
				}

			}
			
		}
		
		
		
		//Set<String> hashSet = lineMap.keySet();
		
		//for (String hash: hashSet)
		{
			//addMap(lineMap.get(hash), lineDistributionMap);
		}
		
		//writeDistributionToFile(messageDstFile, lineDistributionMap);
		
		System.out.println(commitHashSet.size() + " messages referenced " 
				+ bugidReferencedSet.size() + " bugs");
		
	}
	
	private void getMessageStatistics(int rowNumLimt, String targetDstFile) 
			throws SQLException, IOException
	{
		
		
	}
	
	
	
	private void getTargetStatistics(int rowNumLimt, String targetDstFile) 
			throws SQLException, IOException
	{	
		//set up the prepared statement given corresponding sql statements
		String getLinesStatement = "select * from target " +
				"where line is not null group by commitHash";
		getTargetPs = db.getStreamPreparedStatement(getLinesStatement, rowNumLimt);
		
		HashMap<Integer, Integer>targetDistributionMap = 
				new HashMap<Integer, Integer>();
		
		String previousCommitHash = null;
		
		int commitHashNum = 0;
		
		//retrieve bugBody
		ResultSet linesRs = getTargetPs.executeQuery();
		while (linesRs.next())
		{
			
			String commitHash = linesRs.getString(1);
			
			String line = linesRs.getString(2);	
			if (line.equals(previousCommitHash))
			{
				commitHashNum++;
			}
			else
			{
				if (previousCommitHash != null)
				{
					addMap(commitHashNum, targetDistributionMap);
				}
				
				commitHashNum = 1;
			}
			
			previousCommitHash = line;
		}
		
	writeDistributionToFile(targetDstFile, targetDistributionMap);
		
	}
	
	
	/*
	//just get the number of lines in this entity
	private int getTargetStatistics(String commitHash) throws SQLException
	{
		getTargetPs.setString(1, commitHash);
		
		ResultSet targetBodyRs = getTargetPs.executeQuery();
		
		int lineNum = 0;
		while(targetBodyRs.next())
		{
			lineNum++;
		}
		
		//push the size information into map
		addMap(lineNum, targetPDFMap);
		
		return lineNum;
	}
	*/
	
	//quantify and textual analyze this entiry
	private int getMessageStatistics(String messageDstFile) throws SQLException
	{
		//getMessagePs.setString(1, commitHash);
		
		String getMessageStatement = "select * from message";
		getMessagePs = db.getDynamicCheckPreparedStatement(getMessageStatement); 
		
		
		ResultSet messageBodyRs = getMessagePs.executeQuery();
		
		int bugidReferenceCount = 0;
		
		int lineNum = 0;
		while(messageBodyRs.next())
		{
			lineNum++;
			
			//recursively get the line in message
			String line = messageBodyRs.getString(1);
			
			bugidMatch.reset(line);
			while (bugidMatch.find())
			{				
				String idString = bugidMatch.group(bugidMatch.groupCount() - 1);
				
				if (idString != null)
				{
					//lookup the bugid condictionary
					int bugid = Integer.parseInt(idString);
				
					if (bugidSet.contains(bugid))
					{
						bugidReferenceCount++;
						
						bugidReferencedSet.add(bugid);
						
						//commitHashSet.add(commitHash);
								
						System.out.println(bugid + " " + bugidMatch.group(0));
					}
				}

			}
				
		}
		
		//push the size information into map
		addMap(lineNum, messagePDFMap);
		
		return lineNum;
	}
	
	
	
	//add current size into the distribution map
	private void addMap(int size, HashMap<Integer, Integer> distributionMap)
	{
		//if this size exists, just increase it by 1
		if (distributionMap.containsKey(size))
		{
			int num = distributionMap.get(size);
			
			//need to remove it first, inefficient :(
			distributionMap.remove(size);
			
			distributionMap.put(size, num + 1);
		}
		//else add it into this map, set the num as 1
		else
		{
			distributionMap.put(size, 1);
		}
	}
	
	private void addMap(String size, HashMap<String, Integer> distributionMap)
	{
		//if this size exists, just increase it by 1
		if (distributionMap.containsKey(size))
		{
			int num = distributionMap.get(size);
			
			//need to remove it first, inefficient :(
			distributionMap.remove(size);
			
			distributionMap.put(size, num + 1);
		}
		//else add it into this map, set the num as 1
		else
		{
			distributionMap.put(size, 1);
		}
	}
	
	private void loadBugid(int rowNumLimit) throws SQLException
	{
		//store bugid
		bugidSet = new HashSet<Integer>();
		
		//sql statement that counts the number of rows in comment given bugid
		String getBugidStatement = 
				"select distinct bugid from bugBody";
		
		//prepared statement object for comment
		PreparedStatement getBugidps = null;
		
		try
		{
			
			//get the prepared statement
			getBugidps = db.getStreamPreparedStatement(getBugidStatement, 
					rowNumLimit);			
		}
		catch (Exception e)
		{
			e.printStackTrace();
		}
		
		//retrive description
		ResultSet bugidRs = getBugidps.executeQuery();
		while (bugidRs.next())
		{
			int bugid = bugidRs.getInt(1);
		
			bugidSet.add(bugid);
		}

	}
	
	//write the distribution to file
		private void writeDistributionToFile(String dstFile, 
				HashMap<Integer, Integer> distributionMap) throws IOException
		{
			FileWriter fstream = new FileWriter(dstFile);
			
			BufferedWriter out = new BufferedWriter(fstream);
			
			System.out.println(distributionMap.size() + " different distributions");
			
			//heading distribution count
			out.write("# " + distributionMap.size() + "\n"); 
			
			//iteratively push the suffices into file
			
			Set<Integer> sizeSet = distributionMap.keySet();
			
			int totalNum = 0;
			int totalSize = 0;
			
			for (Integer size: sizeSet)
			{
				int num = distributionMap.get(size);
				
				totalNum += num;
				totalSize += size;
				
				out.write(size + "\t" + num + "\n");  
			}
			
			System.out.println("total num --> " + totalNum);
			System.out.println("total size --> " + totalSize);
			
			out.close();
			
		}
		
		public static void main(String args[]) throws ClassNotFoundException, 
		IllegalAccessException, SQLException, IOException
		{
			GetChangeStatistics changeStatics = new GetChangeStatistics();
			
			//changeStatics.getChangeEntityNum("data/changeStatistics.txt",
					//"messageStatistics.txt");
			
			//changeStatics.getLinesStatistics(MESSAGE_ROW_LIMIT, "data/message.txt");
			
			changeStatics.getTargetStatistics(MESSAGE_ROW_LIMIT, "data/target.txt");
		}
}